let cpt = ref 1.

let total = ref 0.

let compact_page in_ch out_ch =
  match Mlib.read_page in_ch with
    | Some title, Some p ->
	let page = Str.full_split (Str.regexp "\\[\\[\\|\\]\\]") p in
	let l = (Mlib.compact [] page) in (* CHANGE LA LIGNE PAR CELLE LA *)
 	let _ = 
	  Mlib.debug "   Compacting %0.4f%%  %0.0f/%0.0f \r" 
	    (100. *. (!cpt)/.(!total)) !cpt !total
	in
	  cpt := !cpt +. 1.;
	let _ = flush stderr in
	(* let _ = Mlib.write_value out_ch (title, l) in () *)
	let _ = Mlib.write out_ch (title ^ (String.concat " " l) ^ "\n") in ()
    | _ -> ()
      
(* Reads a page and returns a set of new word responding to the request *)
let start_compacting in_ch out_ch q = 
  (* XXCAGO XX TODO ici on pourrait lancer les page dans des threads et faire l'union 
     des ensembles à la fin *)
  let rec aux q =
    match q with
      | [] -> () (* End of file *)
      | page_position  :: tl ->
	  let _ = seek_in in_ch page_position in
	  let _ = compact_page in_ch out_ch in
	    aux tl
  in
    aux q

let _ = 
  let offs = open_in Mlib.offsets_list in
  let q : int list = match Mlib.read_value offs with None -> [] | Some l -> l in
    total := float_of_int (List.length q);
  let in_ch  = Mlib.open_in_file Mlib.fr_wiki in
  let out_ch = Mlib.open_out_file Mlib.cwiki in
    Mlib.debug "Start reading %d pages... \n" (List.length q); flush stderr;
    start_compacting in_ch out_ch q
